/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package batch;

import Outils.Outil;
import controleur.MainServlet;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.xpath.XPathExpressionException;
import persistence.Bdd;
import org.w3c.dom.NodeList;

/**
 *
 * @author Nishruu
 */
public class Crawl {

    private static List<String[]> listeSemaines = new ArrayList<String[]>();
    
    private static int diffCrawl;
    private static ExecutorService taskPool = Executors.newFixedThreadPool(16);
    //private ExecutorService taskPool = Executors.newCachedThreadPool();
    private static final String url = "http://ws.audioscrobbler.com/2.0/?method=";
    private static final String api_key = "&api_key=a8447b675add300f70765b79173e69a9";

    public static void main(String args[]) throws UnsupportedEncodingException, Exception {
        //importMetroChartOfWeek("1306670400","1307275200");
        crawlHebdo();
        taskPool.shutdown();
    }
    
    public static void crawlHebdo() {
        //listeSemaines = new ArrayList<String[]>();

        getSemaines();

        if (listeSemaines.size() > 0) {
            for (int compteur = 0; compteur < listeSemaines.size() ; ++compteur ) {
                String debut_semaine = listeSemaines.get(compteur)[0];
                String fin_semaine = listeSemaines.get(compteur)[1];
                
                if(Integer.parseInt(debut_semaine)>= 1304856000) importMetroChartOfWeek(debut_semaine, fin_semaine);
            }
        }
    }

    public static void importMetroChartOfWeek(String debut_semaine, String fin_semaine) {
        List<String[]> listeVilles = getVilles();
        String ville[] = null;
        String nomville = null;
        String pays = null;
        for(int i=0; i < listeVilles.size(); ++i) {
            ville = listeVilles.get(i);
            nomville = ville[0];
            pays = ville[1];

            MetroChartImportTask importTask = new MetroChartImportTask(nomville, pays, debut_semaine, fin_semaine);
            taskPool.submit(importTask);
        }
        System.out.println("Week pooled: " + debut_semaine);
    }

    public static int getSemaines() {
        
        String sql;
        Bdd bd = new Bdd();
        ResultSet rs;
        int diff = 0;

        //Crawl Semaines
        try {

            sql = "SELECT COUNT(*) FROM SEMAINE";
            rs = bd.select(sql);

            rs.next();
            double result = Outil.xParseNumber(url + "geo.getMetroWeeklyChartlist" + api_key, "count(//weeklychartlist/chart)");
            diff = (int) (result - rs.getInt(1));
            rs.close();
            if (diff > 0) {
                System.out.println("a");
                NodeList nodes = Outil.xParseNodeList(url + "geo.getMetroWeeklyChartlist" + api_key, "//weeklychartlist/chart[position() > last()-" + diff + "]/@from|//weeklychartlist/chart[position() > last()-" + diff + "]/@to");
                for (int i = 0; i < nodes.getLength(); i = i + 2) {
                    String[] paire = new String[2];
                    paire[0] = nodes.item(i).getNodeValue();
                    paire[1] = nodes.item(i + 1).getNodeValue();
                    listeSemaines.add(paire);
                    //System.err.println(sql);
                    //sql = "INSERT IGNORE INTO SEMAINE(debut_semaine,fin_semaine) VALUES (?,?);";
                    //bd.update(sql, new String[] {paire[0], paire[1]});
                }
                bd.mass_insert("insert into INTO SEMAINE(debut_semaine,fin_semaine)", listeSemaines);
            }else{
                rs = bd.select("select debut_semaine, fin_semaine from semaine;");
                while(rs.next()) {
                        String[] paire = new String[2];
                        paire[0] = rs.getString(1);
                        paire[1] = rs.getString(2);
                        listeSemaines.add(paire);
                }
                rs.close();
            }
        }
        catch (XPathExpressionException ex) {
            Logger.getLogger(MainServlet.class.getName()).log(Level.SEVERE, null, ex);
        }
        catch (SQLException ex) {
                Logger.getLogger(Crawl.class.getName()).log(Level.SEVERE, null, ex);
        }

        return diff;
    }

    private static NodeList getCoords(String xpath) {
        NodeList nodes = null;
        try {
            nodes = Outil.xParseNodeList(xpath,"//status/text()|//geometry/location/lat/text()|//geometry/location/lng/text()");
        }catch (XPathExpressionException ex) {
            Logger.getLogger(MainServlet.class.getName()).log(Level.SEVERE, null, ex);
        }

        return nodes;
    }

    public static List<String[]> getVilles() {
        Bdd bd = new Bdd();
        List<String[]> listeVilles = new ArrayList<String[]>();
        try {
            ResultSet rs = bd.select("SELECT nom_ville,nom_pays FROM VILLE");
            while (rs.next()) {
                String[] paire = new String[2];
                paire[0] = rs.getString("nom_ville");
                paire[1] = rs.getString("nom_pays");
                listeVilles.add(paire);
            }
        
            NodeList nodes_coords = null;
            String latitude;
            String longitude;
            double result = Outil.xParseNumber(url + "geo.getmetros" + api_key, "count(//metro/name)");
            int diff = (int) (result - listeVilles.size());


            if (diff > 0) {
                NodeList nodes = Outil.xParseNodeList(url + "geo.getmetros" + api_key, "//metro[position() > (last() - " + diff + ")]/name/text()|//metro[position() > (last() - " + diff + ")]/country/text()");
                List<String> villes = new ArrayList<String>();

                for (int i = 0; i < nodes.getLength(); i += 2) {
                    String[] paire = new String[2];
                    paire[0] = nodes.item(i).getNodeValue();
                    paire[1] = nodes.item(i + 1).getNodeValue();

                    String latotale = "http://maps.googleapis.com/maps/api/geocode/xml?address=" + paire[0] + ",+" + paire[1] + "&sensor=false";
                    nodes_coords = getCoords(latotale);

                    try {
                    if (nodes_coords.item(0).getNodeValue().equals("OVER_QUERY_LIMIT")) {
                            Thread.sleep(500);
                            nodes_coords = getCoords(latotale);
                    }else Thread.sleep(50);
                    } catch (InterruptedException ex) {
                            System.err.println("Timer interrompu");
                    }
                    
                    latitude = nodes_coords.item(1).getNodeValue();
                    longitude = nodes_coords.item(2).getNodeValue();
                    listeVilles.add(paire);
                    bd.update("INSERT INTO VILLE(nom_ville,nom_pays,latitude,longitude) VALUES (?,?,?,?)",
                            new Object[] {paire[0],paire[1], latitude, longitude});
                    System.out.println("New city:" + paire[0] + ", " + paire[1] + "[" + latitude + "," + longitude + "]");
                }
            }
        }
        catch (XPathExpressionException ex) {
            Logger.getLogger(Crawl.class.getName()).log(Level.SEVERE, null, ex);
        }
        catch (SQLException ex) {
            Logger.getLogger(Crawl.class.getName()).log(Level.SEVERE, null, ex);
        }
        return listeVilles;
    }


}
